#coding: utf-8

"""
MarketAreas
0       全部
1000    罗湖区
2000    福田区
3000    南山区
4000    盐田区
5000    宝安区
6000    龙岗区

MarketApartment
0       全 部
1       一房一厅
3       两房一厅
4       两房两厅
5       三房一厅
6        三房两厅
7       四房一厅
8       四房两厅
9       五房以上
14      单身公寓
11      商务公寓
12      复式住宅

MarektBuildingDesc

波托菲诺

beautifulsoup
<tr align="center" style="height:22px;background-color:#B1DFDA;"></tr>

<tr align="center" style="background-color:#E8F2F9;height:22px;background-color:#D0ECE9;"></tr>


ctl00$dropDatePublished 0
ctl00$txtSearchKeyWord 波托菲诺
Sell_Search2.aspx?c2VhcmNodHlwZT0wJmRhdGU9MSZrZXl3b3JkPSVlNiViMyVhMiVlNiU4OSU5OCVlOCU4ZiViMiVlOCVhZiViYQ=%3d

c2VhcmNodHlwZT0wJmRhdGU9MSZrZXl3b3JkPSVlNiViMyVhMiVlNiU4OSU5OCVlOCU4ZiViMiVlOCVhZiViYQ=%3d
searchtype=0&date=1&keyword=%e6%b3%a2%e6%89%98%e8%8f%b2%e8%af%ba

c2VhcmNodHlwZT0wJmtleXdvcmQ9JWU2JWIzJWEyJWU2JTg5JTk4JWU4JThmJWIyJWU4JWFmJWJh
searchtype=0&keyword=%e6%b3%a2%e6%89%98%e8%8f%b2%e8%af%ba
urllib.quote(base64.encodestring('searchtype=0&date=1&keyword='+urllib.quote(u'波托菲诺'.encode('utf-8')).lower())[:-1])
"""
import urllib,urllib2
import re
import sys
import datetime
from MySQLdb import connect,IntegrityError
from constants import *

def searchBuildingList (response):
    return re_buildingList.findall(response)

def getPage(url,params=None):
    request = urllib2.Request(url)
    if params:
        request.add_data(params)
    request.add_header('User-Agent','Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)')
    #opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1))
    opener = urllib2.build_opener()
    response = opener.open(request).read()
    return response

def getListPage (url,page_num,VIEWSTATE):
    params=urllib.urlencode([('__EVENTTARGET','ctl00$ContentPlace$gvSellResult'),
                             ('__EVENTARGUMENT','Page$%s'%page_num),
                             ('__VIEWSTATE',VIEWSTATE),
                             ('__VIEWSTATEENCRYPTED','')])
    response=getPage(url,params)
    return searchBuildingList(response)


def getFirstPage (url):
    """params=urllib.urlencode([('__EVENTTARGET','ctl00$ContentPlace$gvSellResult'),
                             ('__EVENTARGUMENT','Page$1'),
                             ('__VIEWSTATE',VIEWSTATE),
                             ('__VIEWSTATEENCRYPTED','')])"""
    response=getPage(url)
    BuildingList=searchBuildingList(response)
    totalPage=int(re_page.search(response).groups()[0])
    VIEWSTATE=re.compile(r'<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="(.*?)" />').search(response).groups()[0]
    return BuildingList,totalPage,VIEWSTATE

def str2date (value):
    date_list=value.split('-')
    if len(date_list)==3:
        date_list=map(int,date_list)
        return datetime.date(*date_list)
    else:
        return None

def getOneDay (whichDay):
    buildingList,totalPage,VIEWSTATE=getFirstPage(all_sell_szhome)
    searchXunPanEnd=False
    for building in buildingList:
        if building[0].strip() and not searchXunPanEnd:#first find the date in the 荀盘
            buildingDate=str2date(building[3])
            if buildingDate>whichDay:
                continue
            elif buildingDate==whichDay:
                print 'fetch the buildng',building[1]
                saveBuilding(building)
            else:
                searchXunPanEnd=True
        elif not building[0].strip():
            buildingDate=str2date(building[3])
            if buildingDate>whichDay:
                continue
            elif buildingDate==whichDay:
                print 'fetch the buildng',building[1]
                saveBuilding(building)
            else:
                continue
    for i in xrange(2,totalPage+1):
        buildingList=getListPage(all_sell_szhome,i,VIEWSTATE)
        isMatch=False
        for building in buildingList:
            if building[0].strip() and not searchXunPanEnd:#first find the date in the 荀盘
                isMatch=True
                buildingDate=str2date(building[3])
                if buildingDate>whichDay:
                    continue
                elif buildingDate==whichDay:
                    print 'fetch the buildng',building[1]
                    saveBuilding(building)
                else:
                    searchXunPanEnd=True
            elif not building[0].strip():
                buildingDate=str2date(building[3])
                if buildingDate>whichDay:
                    isMatch=True
                elif buildingDate==whichDay:
                    isMatch=True
                    print 'fetch the buildng',building[1]
                    saveBuilding(building)
                else:
                    continue
        if not isMatch:
            break


def getToday ():
    buildingList,totalPage,VIEWSTATE=getFirstPage(today_sell_szhome)
    for building in buildingList:
        print 'fetch the building',building[1]
        saveBuilding(building)
    for i in xrange(2,totalPage+1):
        buildingList=getListPage(today_sell_szhome,i,VIEWSTATE)
        for building in buildingList:
            print 'fetch the building',building[1]
            saveBuilding(building)

def getAllDay ():
    buildingList,totalPage,VIEWSTATE=getFirstPage(all_sell_szhome)
    for building in buildingList:
        print 'fetch the buildng',building[1]
        saveBuilding(building)
    for i in xrange(2,totalPage+1):
        buildingList=getListPage(all_sell_szhome,i,VIEWSTATE)
        for building in buildingList:
            print 'fetch the buildng',building[1]
            saveBuilding(building)

def searchDetail (response):
    detailDict=re_detail.search(response).groupdict()
    fitment=[]
    if detailDict['equipment0']:
        fitment.append('水')
    if detailDict['equipment1']:
        fitment.append('电')
    if detailDict['equipment2']:
        fitment.append('管道煤气')
    if detailDict['equipment3']:
        fitment.append('家具')
    if detailDict['equipment4']:
        fitment.append('电话线路')
    if detailDict['equipment5']:
        fitment.append('有线电视线路')
    if detailDict['equipment6']:
        fitment.append('宽带线路')
    if detailDict['cblWiring0']:
        fitment.append('电话')
    if detailDict['cblWiring1']:
        fitment.append('电视')
    if detailDict['cblWiring2']:
        fitment.append('冰箱')
    if detailDict['cblWiring3']:
        fitment.append('热水器')
    if detailDict['cblWiring4']:
        fitment.append('洗衣机')
    if detailDict['cblWiring5']:
        fitment.append('空调')
    detailDict['fitment']=','.join(fitment)
    if detailDict['typeOfFlat']==u'单房'.encode('gbk'):
        detailDict['typeOfFlat']=0
    elif detailDict['typeOfFlat']==u'一房一厅'.encode('gbk'):
        detailDict['typeOfFlat']=1
    elif detailDict['typeOfFlat']==u'两房一厅'.encode('gbk'):
        detailDict['typeOfFlat']=2
    elif detailDict['typeOfFlat']==u'两房两厅'.encode('gbk'):
        detailDict['typeOfFlat']=3
    elif detailDict['typeOfFlat']==u'三房一厅'.encode('gbk'):
        detailDict['typeOfFlat']=4
    else:
        detailDict['typeOfFlat']=5
    if detailDict['title']:
        detailDict['title']=detailDict['title'].decode('gbk').encode('utf-8')
    if detailDict['agent']:
        detailDict['agent']=detailDict['agent'].decode('gbk').encode('utf-8')
    if detailDict['decoration']:
        detailDict['decoration']=detailDict['decoration'].decode('gbk').encode('utf-8')
    if detailDict['exposure']:
        detailDict['exposure']=detailDict['exposure'].decode('gbk').encode('utf-8')
    if detailDict['contactName']:
        detailDict['contactName']=detailDict['contactName'].decode('gbk').encode('utf-8')
    if detailDict['description']: 
        detailDict['description']=detailDict['description'].decode('gbk').encode('utf-8')
    if detailDict['title'] is None:
        detailDict['title']=''
    if detailDict['floor'] is None:
        detailDict['floor']=1
    if detailDict['totalFloors'] is None:
        detailDict['totalFloors']=1
    if detailDict['email'] is None:
        detailDict['email']=''
    if detailDict['yearBuilt'] is None:
        detailDict['yearBuilt']=0
    if detailDict['description'] is None:
        detailDict['description']=''
    return detailDict

def getDetail (msgid):
    response=getPage(detail_sell_szhome%msgid)
    if response.find((u'该信息已经已经被禁用'.encode('gbk')))!=-1:
        if cursor.execute('select id from building where originalUrl=%s',detail_sell_szhome%msgid):
            tus=cursor.fetchall()
            for tu in tus:
                cursor.execute('delete from buildingPrice where building_id=%s',tu[0])
            cursor.execute('delete from building where id=%s',tu[0])
        return {}
    else:
        return searchDetail(response)

def saveBuilding (building):
    try:
        cursor.execute('insert into building (leaseSale,housingProject_id,purpose,originalUrl) values (2,1,1,%s)','http://market.szhome.com/Sell_View.aspx?MessageID=%s'%building[1])
    except IntegrityError:
        cursor.execute('select id from building where originalUrl=%s','http://market.szhome.com/Sell_View.aspx?MessageID=%s'%building[1])
        building_id=cursor.fetchone()[0]
    else:
        building_id=cursor.lastrowid
        buildingDict=getDetail(building[1])
        cursor.execute('update building set title=%s,squareFootage=%s,typeOfFlat=%s,yearBuilt=%s,fitment=%s,floor=%s,totalFloors=%s,agent=%s,phoneNum=%s,contactName=%s,email=%s,decoration=%s,exposure=%s,description=%s where id=%s',(buildingDict['title'],buildingDict['squareFootage'],buildingDict['typeOfFlat'],buildingDict['yearBuilt'],buildingDict['fitment'],buildingDict['floor'],buildingDict['totalFloors'],buildingDict['agent'],buildingDict['phoneNum'],buildingDict['contactName'],buildingDict['email'],buildingDict['decoration'],buildingDict['exposure'],buildingDict['description'],building_id))
    postDate=str2date(building[3])
    limitDate=postDate+datetime.timedelta(15)
    cursor.execute('insert into buildingprice (building_id,price,postDate,limitDate) values (%s,%s,%s,%s) ON DUPLICATE KEY UPDATE price=%s,postDate=%s,limitDate=%s',(building_id,building[2],postDate,limitDate,building[2],postDate,limitDate))

import time
if __name__=='__main__':
    #print getDetail(3758223)['fitment'].decode('utf-8').encode('gbk')
    #sys.exit(0)
    conn=connect(user='cheng3',passwd='123456',db='cheng3',host='localhost')
    cursor=conn.cursor()
    cursor.execute('set names utf8')
    cursor.execute('set character set utf8')
    print 'Begin to fetch data'
    if sys.argv[1:]:
        if sys.argv[1]=='--all':
            getAllDay()
        else:
            whichDay=str2date(sys.argv[1])
            if whichDay:
                getOneDay(whichDay)
    else:
        getToday()
    cursor.close()
    conn.close()
    print "ByeBye!"
